{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "DEBUG:jieba:Building prefix dict from the default dictionary ...\n",
      "Loading model from cache c:\\users\\warren\\appdata\\local\\temp\\jieba.cache\n",
      "DEBUG:jieba:Loading model from cache c:\\users\\warren\\appdata\\local\\temp\\jieba.cache\n",
      "Loading model cost 0.684 seconds.\n",
      "DEBUG:jieba:Loading model cost 0.684 seconds.\n",
      "Prefix dict has been built succesfully.\n",
      "DEBUG:jieba:Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "中国/是/世界/上/人口/最多/的/国家/，/有/悠久/的/文明/和/历史\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "\n",
    "str='中国是世界上人口最多的国家，有悠久的文明和历史'\n",
    "a=jieba.cut(str,cut_all=False)\n",
    "print '/'.join(a)\n",
    "# 文本文件：\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.models.word2vec:under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "读取文件耗时： 3510.01931434\n",
      "转换后:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 参考地址：http://www.52nlp.cn/%E4%B8%AD%E8%8B%B1%E6%96%87%E7%BB%B4%E5%9F%BA%E7%99%BE%E7%A7%91%E8%AF%AD%E6%96%99%E4%B8%8A%E7%9A%84word2vec%E5%AE%9E%E9%AA%8C\n",
    "import logging  \n",
    "import os  \n",
    "import time  \n",
    "  \n",
    "import gensim  \n",
    "from gensim.models import word2vec  \n",
    "import jieba  \n",
    "#import nltk \n",
    "import json\n",
    "  \n",
    "logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',level=logging.INFO)    \n",
    "start1 = time.clock()   \n",
    "input_file_name = u'E:/百度云/IT技术_new/编程语言/python/demo/word/result.txt' # 原始文件Unicode编码\n",
    "input_file_f = open(input_file_name,'r')  \n",
    "#contents = input_file_f.read() # 整个文件读到一个变量里\n",
    "print '读取文件耗时：',time.clock()\n",
    "#sentences = [i.strip().split(\" \") for  i in contents[:10]]\n",
    "sentences = []\n",
    "print '转换后:\\n','|'.join(['&'.join(i) for i in sentences])\n",
    "# 开始逐行处理\n",
    "for line in input_file_f.readlines(): \n",
    "    #按行读取 \n",
    "    sentences.append(line.strip().split(\" \"))\n",
    "#print '行数:%s,内容:\\n'%(len(sentences)),json.dumps(sentences,ensure_ascii=False)\n",
    "#sentences是句子序列，句子又是单词列表，比如，sentences = [['first', 'sentence'], ['second', 'sentence']]\n",
    "model = word2vec.Word2Vec(sentences,min_count=2,size=200) #min_count表示小于该数的单词会被剔除，默认值为5;size表示神经网络的隐藏层单元数，默认为100\n",
    "#保存生成的训练模型\n",
    "output_model = u'E:/百度云/IT技术_new/编程语言/python/demo/word/model'\n",
    "model.save(output_model)#加载模型文件new_model = gensim.models.Word2Vec.load('model/mymodel4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[\"数据\", 0.2086765170097351], [\"希望\", 0.2059345245361328], [\"资料\", 0.19669251143932343], [\"效率\", 0.19636818766593933], [\"世界\", 0.17260316014289856], [\"兑现\", 0.1699996292591095], [\"重复\", 0.1614733636379242], [\"效果\", 0.159501850605011], [\"成\", 0.15193060040473938], [\"提供\", 0.15176695585250854]]\n",
      "（1）找某个词的相似词汇如下:\n",
      "词汇\t相似度\n",
      "数据\t0.20867651701\n",
      "希望\t0.205934524536\n",
      "资料\t0.196692511439\n",
      "效率\t0.196368187666\n",
      "世界\t0.172603160143\n",
      "兑现\t0.169999629259\n",
      "重复\t0.161473363638\n",
      "效果\t0.159501850605\n",
      "成\t0.151930600405\n",
      "提供\t0.151766955853\n",
      "（2）任意两个词汇的相似度(经理与数据) 0.208676512331\n",
      "（3）两个数据集间的余弦距离([\"经理\", \"效率\"])与([\"经理\", \"效率\"])： 0.18038151507\n",
      "（4）找集合中不同的一项：([\"数据\", \"流程\", \"重复\"]) 数据\n"
     ]
    }
   ],
   "source": [
    "#加载模型文件\n",
    "new_model = gensim.models.Word2Vec.load(output_model)\n",
    "dir(new_model) # 多种函数方法,\n",
    "print new_model.vector_size # 词向量维度\n",
    "print ','.join(new_model.index2word) # index2word保存单词\n",
    "# 计算指定词的所以相似词\n",
    "test_word = '经理'\n",
    "similar_word_list = new_model.most_similar(test_word)\n",
    "print json.dumps(similar_word_list,ensure_ascii=False)\n",
    "#print json.dumps(similar_word_list,ensure_ascii=False,indent=4)\n",
    "# 抽取北京的搜索session：select query_list from user_satisfy_query where dt=20160918 and province rlike '^010' and count > 1;\n",
    "#print json.dumps(new_model.most_similar(u'天安门'),ensure_ascii=False)\n",
    "#In [76]: print json.dumps(new_model.most_similar(u'旅店'),ensure_ascii=False)\n",
    "#[[\"莫泰\", 0.8472937345504761], [\"易佰\", 0.8139138221740723], [\"168\", 0.7009128928184509], [\"连锁\", 0.6979336738586426], [\"旅馆\", 0.6874777674674988], [\"旺子成\", 0.6520262360572815], [\"快捷\", 0.6426747441291809], [\"家庭旅馆\", 0.6317397356033325], [\"人在旅途\", 0.6164605021476746], [\"寺易佰\", 0.6112728714942932]]\n",
    "#In [77]: print json.dumps(new_model.most_similar(u'菜馆'),ensure_ascii=False)\n",
    "#[[\"家常菜\", 0.8295753598213196], [\"风味\", 0.8144116401672363], [\"正宗\", 0.8008058071136475], [\"菜\", 0.787124514579773], [\"饺子馆\", 0.7830443382263184], [\"刀削面\", 0.7752013802528381], [\"特色\", 0.7629570364952087], [\"面馆\", 0.7591361403465271], [\"面\", 0.7421250939369202], [\"农家菜\", 0.7410575747489929]]\n",
    "#In [158]: print json.dumps(new_model.most_similar(u'软件园'),ensure_ascii=False)  \n",
    "#[[\"用友\", 0.7017531991004944], [\"金蝶\", 0.6142528057098389], [\"孵化器\", 0.5947192907333374], [\"网易\", 0.5910834074020386], [\"f11\", 0.584527850151062], [\"软件\", 0.5816747546195984], [\"租贷\", 0.5489269495010376], [\"卵\", 0.5268262624740601], [\"鲜花网\", 0.5116425156593323], [\"广联达\", 0.507921576499939]]\n",
    "#In [171]: print json.dumps(new_model.most_similar(u'美食'),ensure_ascii=False)\n",
    "#[[\"中餐\", 0.8337364196777344], [\"川菜\", 0.7456749677658081], [\"快餐\", 0.7315336465835571], [\"西餐\", 0.6596412658691406], [\"自助餐\", 0.6401817202568054], [\"老姬\", 0.6020432710647583], [\"日本料理\", 0.5849108099937439], [\"合利屋\", 0.5827316045761108], [\"nokia\", 0.5804284811019897], [\"早点\", 0.5785887241363525]]\n",
    "#In [176]: print json.dumps(new_model.most_similar(u'麦当劳'),ensure_ascii=False)\n",
    "#[[\"肯德基\", 0.857654869556427], [\"肯德鸡\", 0.6457746028900146], [\"KFC\", 0.6434839963912964], [\"kfc\", 0.6308714151382446], [\"街鼎\", 0.6141167283058167], [\"FSDT\", 0.589178204536438], [\"康得基\", 0.5770742893218994], [\"得来\", 0.5747169852256775], [\"十佛营\", 0.5702893137931824], [\"必胜客\", 0.5698955655097961]]\n",
    "print '（1）找某个词的相似词汇如下:\\n词汇\\t相似度\\n','\\n'.join(['%s\\t%s'%(i[0],i[1]) for i in similar_word_list])\n",
    "# 计算任意两个词的相似度\n",
    "word_1 = '经理';word_2 = '数据'\n",
    "print '（2）任意两个词汇的相似度(%s与%s)'%(word_1,word_2),new_model.similarity(word_1,word_2)\n",
    "word_set_1 = ['经理','效率'];word_set_2 = ['数据','流程','重复']\n",
    "print '（3）两个数据集间的余弦距离(%s)与(%s)：'%(json.dumps(word_set_1,ensure_ascii=False),json.dumps(word_set_1,ensure_ascii=False)),new_model.n_similarity(word_set_1, word_set_2) \n",
    "print '（4）找集合中不同的一项：(%s)'%(json.dumps(word_set_2,ensure_ascii=False)),new_model.doesnt_match(word_set_2)\n",
    "# 独特的组合加减法\n",
    "print json.dumps(new_model.most_similar(positive=[u'麦当劳'],negative=[u'肯德基',u'真功夫']),ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 驱动器 C 中的卷是 Windows\n",
      " 卷的序列号是 9CE0-E290\n",
      "\n",
      " C:\\Users\\wqw\\Desktop\\code\\mycode\\wangqiwen\\jupyter 的目录\n",
      "\n",
      "2018/06/26  23:08    <DIR>          .\n",
      "2018/06/26  23:08    <DIR>          ..\n",
      "2018/06/26  22:37    <DIR>          .ipynb_checkpoints\n",
      "2018/04/01  14:24           710,932 cluster_job.ipynb\n",
      "2018/04/01  14:24            15,406 draw.ipynb\n",
      "2018/04/01  14:24            16,076 model.ipynb\n",
      "2018/06/26  23:08           244,391 scikit-learn-example.ipynb\n",
      "2018/04/01  14:24           101,886 test_pandas_numpy.ipynb\n",
      "2018/06/26  22:28            92,224 text_process.ipynb\n",
      "2018/04/01  14:24            13,321 word2vec.ipynb\n",
      "               7 个文件      1,194,236 字节\n",
      "               3 个目录  7,475,970,048 可用字节\n"
     ]
    }
   ],
   "source": [
    "!dir ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
